import scrapy
from scrapy import FormRequest
from scrapy.http import JsonRequest
from scrapy import Spider
from items.ugitems import SongItem,DetailItem
import json 
import re 

'''
爬取 Ultimate Guitar 网站的歌手信息
command: scrapy crawl ug-search-spider -a title="Ed Sheeran+Perfect" -o output.csv
'''
class UgSearchSpider(scrapy.Spider):
    name = 'ug-search-spider'
    # 查询条件
    title= ''
    page = 1
    loginUrl = 'https://www.ultimate-guitar.com/user/auth/processSignIn'
    start_urls = []

    custom_settings = {
        'ITEM_PIPELINES': {
            'pipelines.ugpipeline.ArtistPipeline': 2,
            'pipelines.ugpipeline.DetailPipeline': 3,
            'pipelines.images.AlbumImgPipeline': 4,
            'pipelines.images.ArtistImgPipeline': 5,
            'pipelines.csv.CsvExportPipeline': 6,
        }
    }   

    def __init__(self, name = None, **kwargs):
        super().__init__(name, **kwargs)
        self.logger.debug('params==>'+str(kwargs))
        self.title = kwargs.get('title')
        self.start_urls = [
            'https://www.ultimate-guitar.com/search.php?title=' + self.title + '&page='+str(self.page) +'&rating%5B0%5D=4&rating%5B1%5D=5&order=myweight&type=300'
        ]

    def parse(self, response):
        self.logger.debug('self.start_Urls: ' + self.start_urls[0])        
        # try:
        jsstore = response.xpath('//div[@class="js-store"]').xpath('//div/@data-content').extract()[0]
        if(jsstore):
            jsstore = json.loads(jsstore)
            pagination = jsstore["store"]["page"]["data"]["pagination"]
            total = pagination["total"] 
            print("total: " + str(total)+"; pagination: " + str(pagination)+";")
            items = jsstore["store"]["page"]["data"]["results"]
            yield from self.parse_next_page(response,total) # 递归调用
            for item in items:
                if(item.__contains__("type")):
                    if(item['type']=="Chords"):
                        song = SongItem(
                            id=item["id"],
                            song_id=item["song_id"],
                            song_name=item["song_name"],
                            artist_id=item["artist_id"],
                            artist_name=item["artist_name"],
                            type=item["type"],
                            album_cover=self.getAlbumCover(item),
                            artist_cover=self.getArtistCover(item),
                            artist_url=item["artist_url"],
                            tab_url=item["tab_url"],
                        )
                        yield song
                        #请示详情页
                        yield response.follow(url=item["tab_url"], callback=self.parse_detail,meta=dict(song))
        # except Exception as e:
        #     self.logger.error('抓取网页时出错了：'+e.__str__())

    '''
    请求下一页
    '''
    def parse_next_page(self, response, total):    
        self.logger.debug('当前第'+str(self.page)+'页')
        self.page = self.page + 1
        self.logger.debug('self.page='+str(self.page))
        if(self.page <= total):
            self.logger.debug('正在爬取第'+str(self.page)+'页,total:'+str(total))
            next_page = 'https://www.ultimate-guitar.com/search.php?title=' + self.title + '&page='+str(self.page) +'&rating%5B0%5D=4&rating%5B1%5D=5&order=myweight&type=300'
            self.logger.debug('next_page: ' + next_page)
            yield response.follow(next_page, callback=self.parse)

    """
    解析详情页
    """
    def parse_detail(self, response):
        jsstore = response.xpath('//div[@class="js-store"]').xpath('//div/@data-content').extract()[0]
        if(jsstore):
            jsstore = json.loads(jsstore)
            tabContent = jsstore["store"]["page"]["data"]["tab_view"]["wiki_tab"] ["content"]
            detailItem = DetailItem(
                song_id=response.meta["song_id"],
                song_name=response.meta["song_name"],
                artist_id=response.meta["artist_id"],
                artist_name=response.meta["artist_name"],
                type=response.meta["type"],
                tab_content=tabContent,
            )
        yield detailItem


    def getAlbumCover(self,item):
        print('getAlbumCover=>'+item.__str__() +'; type:'+str(type(item)))
        album_cover = item['album_cover']
        if(album_cover):
            has_album_cover = album_cover['has_album_cover']
            if(has_album_cover):
                img = album_cover["web_album_cover"]["small"]
                if(img):
                    return img 
                else:
                    return ''
        else:
            return ''

    def getArtistCover(self,item):
        artist_cover = item['artist_cover']
        if(artist_cover):
            return artist_cover["web_artist_cover"]["small"] 
        else:
            return ''        
